# -*- coding: utf-8 -*-
"""
******* 文档说明 ******
MoviesLen数据预处理部分

# 当前项目: DSSM_Rec
# 创建时间: 2020-08-20  12:04
# 开发作者: Vincent
# 版    本: V1.0
"""
import os
import sys
import numpy as np
import pandas as pd
import tensorflow as tf
import re

np.set_printoptions(precision=3)  # 设置 numpy 显示位数
print('Platform : {} [win32/linux]'.format(sys.platform))  # 当前平台信息
print('Systerm  : {} '.format(sys.version))
print('numpy  Version: {}'.format(np.__version__))
print('pandas Version: {}'.format(pd.__version__))
print('tensorflow Version: {}'.format(tf.__version__))


def load_data(data_path, debug=False):
    """
    Load Dataset from File
    """
    # ###############################　　读取User数据
    users_title = ['UserID', 'Gender', 'Age', 'JobID', 'Zip-code']
    users = pd.read_table(os.path.join(data_path, 'users.dat'), sep='::', header=None, names=users_title,
                          engine='python')
    users = users.filter(regex='UserID|Gender|Age|JobID')  # 去除 Zip-code 列数据
    users_orig = users.values

    # 改变User数据中性别和年龄
    gender_map = {'F': 0, 'M': 1}
    users['Gender'] = users['Gender'].map(gender_map)

    age_map = {val: ii for ii, val in enumerate(sorted(set(users['Age'])))}  # 年龄数据加排序
    # age_map = {val: ii for ii, val in enumerate(set(users['Age']))}  # TODO 加个排序更好？
    users['Age'] = users['Age'].map(age_map)

    if debug:
        print('{}{:^20s}{}'.format('-' * 30, 'User Data', '-' * 30))
        print('Gender Map: {}\n'.format(gender_map))
        print('Age    Map: {}\n'.format(age_map))
        print('users Head: {}\n'.format(users.head()))

    # ###############################　　读取Movie数据集
    movies_title = ['MovieID', 'Title', 'Genres']
    movies = pd.read_table(os.path.join(data_path, 'movies.dat'), sep='::', header=None, names=movies_title,
                           engine='python')
    movies_orig = movies.values

    # 电影类型转数字字典
    genres_set = set()
    for val in movies['Genres'].str.split('|'):
        genres_set.update(val)

    genres_set.add('<PAD>')
    genres2int = {val: ii for ii, val in enumerate(sorted(genres_set))}  # 加上排序

    # 将电影类型转成等长数字列表，长度是18
    genres_map = {val: [genres2int[row] for row in val.split('|')] for ii, val in enumerate(set(movies['Genres']))}

    for key in genres_map:
        for cnt in range(max(genres2int.values()) - len(genres_map[key])):
            genres_map[key].insert(len(genres_map[key]) + cnt, genres2int['<PAD>'])

    movies['Genres'] = movies['Genres'].map(genres_map)

    # 将Title中的年份去掉
    pattern = re.compile(r'^(.*)\((\d+)\)$')  # TODO 为啥 去掉年份

    title_map = {val: pattern.match(val).group(1) for ii, val in enumerate(set(movies['Title']))}
    movies['Title'] = movies['Title'].map(title_map)

    # 电影Title转数字字典
    title_set = set()
    for val in movies['Title'].str.split():  # TODO 电影名称 拆分 成字符？
        title_set.update(val)

    title_set.add('<PAD>')
    title_set = sorted(title_set)  # 加上排序
    title2int = {val: ii for ii, val in enumerate(title_set)}

    # 将电影Title转成等长数字列表，长度是15
    title_count = 15
    title_map = {val: [title2int[row] for row in val.split()] for ii, val in enumerate(set(movies['Title']))}

    for key in title_map:
        for cnt in range(title_count - len(title_map[key])):
            title_map[key].insert(len(title_map[key]) + cnt, title2int['<PAD>'])

    movies['Title'] = movies['Title'].map(title_map)

    if debug:
        print('{}{:^20s}{}'.format('-' * 30, 'Movies Data', '-' * 30))
        print('movies.Title  len:【{}】 Head: \n{}\n'.format(len(movies['Title'][0]), movies['Title'].head()))
        print('genres2int: {}\n'.format(genres2int))
        print('movies.Genres len:【{}】 Head: \n{}\n'.format(len(movies['Genres'][0]), movies['Genres'].head()))

    # ###############################　　读取评分数据集
    ratings_title = ['UserID', 'MovieID', 'ratings', 'timestamps']
    ratings = pd.read_table(os.path.join(data_path, 'ratings.dat'), sep='::', header=None, names=ratings_title,
                            engine='python')
    ratings = ratings.filter(regex='UserID|MovieID|ratings')

    # 合并三个表
    data = pd.merge(pd.merge(ratings, users), movies)

    # 将数据分成X和y两张表
    target_fields = ['ratings']
    features_pd, targets_pd = data.drop(target_fields, axis=1), data[target_fields]

    features = features_pd.values
    targets_values = targets_pd.values

    if debug:
        print('title_count    电影名称转换长度:{}'.format(title_count))
        print('title_set      电影名称转换列表')
        print('genres2int     电影类型转换字典')

        print('features       转换后特征数据（User+Movie）  Shape:【{}】'.format(features.shape))
        print('targets_values 评分Label  Shape:【{}】'.format(targets_values.shape))

        print('ratings 评分 Shape:【{}】'.format(ratings.shape))
        print('users   用户 Shape:【{}】'.format(users.shape))
        print('movies  电影 Shape:【{}】'.format(movies.shape))

        print('data  合并三个表后数据  Shape:【{}】'.format(data.shape))
        print('movies_orig  电影原始数据 Shape:【{}】'.format(movies_orig.shape))
        print('users_orig   用户原始数据 Shape:【{}】'.format(users_orig.shape))
    return title_count, title_set, genres2int, features, targets_values, ratings, users, movies, data, movies_orig, users_orig


def main():
    data_path = os.path.abspath(os.path.join(os.path.dirname(__file__), 'data', 'ml-1m'))
    # 数据处理
    title_count, title_set, genres2int, features, targets_values, ratings, users, movies, data, movies_orig, \
        users_orig = load_data(data_path, debug=True)

    import pickle

    os.makedirs(os.path.join(os.path.dirname(__file__), '_temp'), exist_ok=True)
    # 保存处理好数据
    pickle.dump((title_count, title_set, genres2int, features, targets_values, ratings, users, movies, data,
                 movies_orig,   users_orig),
                open(os.path.join(os.path.dirname(__file__), '_temp', 'preprocess_data.p'), 'wb'))


if __name__ == '__main__':
    main()
    print(os.path.abspath(os.path.dirname(__file__)))
